Home Explore Help
Register Sign In
J1X
/
huginn
Watch 2
Star 0
Fork 0

Browse Source
Add a Liquid filter `uri_expand`

This implements #853.

Akinori MUSHA 9 years ago
parent
1e57f3a457
commit
f0a13f778b
3 changed files with 105 additions and 0 deletions
Show Diff Stats
  1. 1 0
      CHANGES.md
  2. 36 0
      app/concerns/liquid_interpolatable.rb
  3. 68 0
      spec/concerns/liquid_interpolatable_spec.rb

+ 1 - 0
CHANGES.md
View File 

@@ -1,5 +1,6 @@
1 1 
 # Changes
2 2
3 
+* Jun 15, 2015   - Liquid filter `uri_expand` added.
3 4 
 * Jun 12, 2015   - RSSAgent can now accept an array of URLs.
4 5 
 * Jun 8, 2015    - WebsiteAgent includes a `use_namespaces` option to enable XML namespaces.
5 6 
 * May 27, 2015   - Validation warns user if they have not provided a `path` when using JSONPath in WebsiteAgent.

+ 36 - 0
app/concerns/liquid_interpolatable.rb
View File 

@@ -132,6 +132,42 @@ module LiquidInterpolatable
132 132 
       nil
133 133 
     end
134 134
135 
+    # Get the destination URL of a given URL by recursively following
136 
+    # redirects, up to 5 times in a row.  If a given string is not a
137 
+    # valid absolute HTTP URL, or any error occurs while following
138 
+    # redirects, the original string is returned.
139 
+    def uri_expand(url, limit = 5)
140 
+      uri = URI(url)
141 
+
142 
+      http = Faraday.new do |builder|
143 
+        builder.adapter :net_http
144 
+        # builder.use FaradayMiddleware::FollowRedirects, limit: limit
145 
+        # ...does not handle non-HTTP URLs.
146 
+      end
147 
+
148 
+      limit.times do
149 
+        begin
150 
+          case uri
151 
+          when URI::HTTP
152 
+            response = http.head(uri)
153 
+            case response.status
154 
+            when 301, 302, 303, 307
155 
+              if location = response['location']
156 
+                uri += location
157 
+                next
158 
+              end
159 
+            end
160 
+          end
161 
+        rescue
162 
+        end
163 
+
164 
+        return uri.to_s
165 
+      end
166 
+
167 
+      # too many redirections
168 
+      url
169 
+    end
170 
+
135 171 
     # Escape a string for use in XPath expression
136 172 
     def to_xpath(string)
137 173 
       subs = string.to_s.scan(/\G(?:\A\z|[^"]+|[^']+)/).map { |x|

+ 68 - 0
spec/concerns/liquid_interpolatable_spec.rb
View File 

@@ -96,4 +96,72 @@ describe LiquidInterpolatable::Filters do
96 96 
       expect(@agent.interpolated['foo']).to eq('/dir/foo/index.html')
97 97 
     end
98 98 
   end
99 
+
100 
+  describe 'uri_expand' do
101 
+    before do
102 
+      stub_request(:head, 'https://t.co.x/aaaa').
103 
+        to_return(status: 301, headers: { Location: 'https://bit.ly.x/bbbb' })
104 
+      stub_request(:head, 'https://bit.ly.x/bbbb').
105 
+        to_return(status: 301, headers: { Location: 'http://tinyurl.com.x/cccc' })
106 
+      stub_request(:head, 'http://tinyurl.com.x/cccc').
107 
+        to_return(status: 301, headers: { Location: 'http://www.example.com/welcome' })
108 
+
109 
+      (1..5).each do |i|
110 
+        stub_request(:head, "http://2many.x/#{i}").
111 
+          to_return(status: 301, headers: { Location: "http://2many.x/#{i+1}" })
112 
+      end
113 
+      stub_request(:head, 'http://2many.x/6').
114 
+        to_return(status: 301, headers: { 'Content-Length' => '5' })
115 
+    end
116 
+
117 
+    it 'should follow redirects' do
118 
+      expect(@filter.uri_expand('https://t.co.x/aaaa')).to eq('http://www.example.com/welcome')
119 
+    end
120 
+
121 
+    it 'should respect the limit for the number of redirects' do
122 
+      expect(@filter.uri_expand('http://2many.x/1')).to eq('http://2many.x/1')
123 
+      expect(@filter.uri_expand('http://2many.x/1', 6)).to eq('http://2many.x/6')
124 
+    end
125 
+
126 
+    it 'should detect a redirect loop' do
127 
+      stub_request(:head, 'http://bad.x/aaaa').
128 
+        to_return(status: 301, headers: { Location: 'http://bad.x/bbbb' })
129 
+      stub_request(:head, 'http://bad.x/bbbb').
130 
+        to_return(status: 301, headers: { Location: 'http://bad.x/aaaa' })
131 
+
132 
+      expect(@filter.uri_expand('http://bad.x/aaaa')).to eq('http://bad.x/aaaa')
133 
+    end
134 
+
135 
+    it 'should be able to handle an FTP URL' do
136 
+      stub_request(:head, 'http://downloads.x/aaaa').
137 
+        to_return(status: 301, headers: { Location: 'http://downloads.x/download?file=aaaa.zip' })
138 
+      stub_request(:head, 'http://downloads.x/download').
139 
+        with(query: { file: 'aaaa.zip' }).
140 
+        to_return(status: 301, headers: { Location: 'ftp://downloads.x/pub/aaaa.zip' })
141 
+
142 
+      expect(@filter.uri_expand('http://downloads.x/aaaa')).to eq('ftp://downloads.x/pub/aaaa.zip')
143 
+    end
144 
+
145 
+    describe 'used in interpolation' do
146 
+      before do
147 
+        @agent = Agents::InterpolatableAgent.new(name: "test")
148 
+      end
149 
+
150 
+      it 'should follow redirects' do
151 
+        @agent.interpolation_context['short_url'] = 'https://t.co.x/aaaa'
152 
+        @agent.options['long_url'] = '{{ short_url | uri_expand }}'
153 
+        expect(@agent.interpolated['long_url']).to eq('http://www.example.com/welcome')
154 
+      end
155 
+
156 
+      it 'should respect the limit for the number of redirects' do
157 
+        @agent.interpolation_context['short_url'] = 'http://2many.x/1'
158 
+        @agent.options['long_url'] = '{{ short_url | uri_expand }}'
159 
+        expect(@agent.interpolated['long_url']).to eq('http://2many.x/1')
160 
+
161 
+        @agent.interpolation_context['short_url'] = 'http://2many.x/1'
162 
+        @agent.options['long_url'] = '{{ short_url | uri_expand:6 }}'
163 
+        expect(@agent.interpolated['long_url']).to eq('http://2many.x/6')
164 
+      end
165 
+    end
166 
+  end
99 167 
 end